def remove_duplicates(input_file, output_file):
    with open(input_file, 'r', encoding='utf-8') as file:
        unique_lines = set(file.readlines())

    with open(output_file, 'w', encoding='utf-8') as file:
        file.writelines(unique_lines)

# Usage
input_csv = './stages/error_50_1.csv'  # Replace with your input file path
output_csv = './stages/stage_50_1.csv'  # Replace with your desired output file path
remove_duplicates(input_csv, output_csv)
